package com.reader.spider.service;

import com.reader.spider.SevenKanKanDBPipeline;
import com.reader.spider.SevenKanKanProcessor;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

@Service
public class SevenKanKanSpiderService implements Pipeline, PageProcessor {

    Spider spider = Spider.create(new SevenKanKanProcessor());

    String continueQueryAllBookInfo(String pageIndex) {

        return "";
    }

    String continueQueryBookSubjuects(String bookId) {

        return "";
    }

    String queryAllBookInfo(String pageIndex) {

        return "";
    }


    String refreshBookInfo(String bookId) {

        return "";
    }


    String querySubjuects(String bookId) {

        return "";
    }


    String startSipder(String bookID,String subjectID) {

        return "";
    }

    /*******   Pipeline  *******/
    @Override
    public void process(ResultItems resultItems, Task task) {
        System.out.println("resultItems"+resultItems.toString());
    }


    /*******   PageProcessor  *******/
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(http://m.7kankan.com/[0-9]+.html)").all());
        page.addTargetRequests(page.getHtml().links().regex("(http://m.7kankan.com/xiaoshuotopallvote/[0-9]+/[0-9]+.html)").all());
        page.putField("url", page.getUrl().regex("(http://m.7kankan.com/[0-9]+.html)").toString());
        page.putField("name", page.getHtml().xpath("//div[@class='block']/div[@class='block_txt2']/h2/a/text()").toString());
        if (page.getResultItems().get("name")==null || page.getResultItems().get("url")==null){
            //skip this page
            page.setSkip(true);
        }
        page.putField("intoduceInfo", page.getHtml().xpath("//div[@class='cover']/div[@class='intro_info']/tidyText()"));
        page.putField("catalogueUrl", page.getHtml().links().regex("(http://m.7kankan.com/html/[0-9]+/[0-9]+/index.html)").all());
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider spider = Spider.create(new SevenKanKanProcessor());
        spider.clearPipeline();
        spider.addUrl("http://m.7kankan.com/xiaoshuotopallvote/0/1.html").addPipeline(new SevenKanKanDBPipeline()).thread(5).run();
    }
}
